
#importing Labiraires 
import pandas as pd 
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression,Ridge, RidgeCV
from sklearn.metrics import r2_score, mean_squared_error
import numpy as np 
import matplotlib.pyplot as plt 
import seaborn as sns 
%matplotlib inline
from wordcloud import WordCloud


#reading file 
calendar = pd.read_csv ('calendar.csv')
calendar.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 1308890 entries, 0 to 1308889
Data columns (total 4 columns):
 #   Column      Non-Null Count    Dtype 
---  ------      --------------    ----- 
 0   listing_id  1308890 non-null  int64 
 1   date        1308890 non-null  object
 2   available   1308890 non-null  object
 3   price       643037 non-null   object
dtypes: int64(1), object(3)
memory usage: 39.9+ MB


calendar['date'] = pd.to_datetime(calendar['date']).dt.normalize()


calendar['listing_id'] = calendar['listing_id'].astype(str)


calendar['price'] = calendar['price'].replace({'\$':''}, regex = True)


calendar.price.notnull().value_counts()

False    665853
True     643037
Name: price, dtype: int64


calendar.available.value_counts()

f    665853
t    643037
Name: available, dtype: int64


calendar.dropna(subset = ['price'] , inplace= True )


calendar['price'] = pd.to_numeric(calendar['price'],errors='coerce')


calendar.head(20)


calendar.date.max () ,  calendar.date.min()

(Timestamp('2017-09-05 00:00:00'), Timestamp('2016-09-06 00:00:00'))


calendar.info()

<class 'pandas.core.frame.DataFrame'>
Int64Index: 643037 entries, 365 to 1308879
Data columns (total 4 columns):
 #   Column      Non-Null Count   Dtype         
---  ------      --------------   -----         
 0   listing_id  643037 non-null  object        
 1   date        643037 non-null  datetime64[ns]
 2   available   643037 non-null  object        
 3   price       640469 non-null  float64       
dtypes: datetime64[ns](1), float64(1), object(2)
memory usage: 24.5+ MB


calendar['quarter'] = pd.PeriodIndex(calendar.date, freq='Q')


calendar['month'] = pd.PeriodIndex(calendar.date, freq='m')


plt.rcParams["figure.figsize"] = (20,8)
sns.boxplot( data = calendar  , x='month',  y = 'price' )

<AxesSubplot:xlabel='month', ylabel='price'>


plt.rcParams["figure.figsize"] = (20,8)
sns.distplot(calendar['price'], hist=True, kde=True, 
             bins=int(180/5), color = 'darkblue', 
             hist_kws={'edgecolor':'black'},
             kde_kws={'linewidth': 4})
plt.title ('disrtibution of price over time')

C:\Users\eljaz\anaconda3\lib\site-packages\seaborn\distributions.py:2551: FutureWarning:

`distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `histplot` (an axes-level function for histograms).

Text(0.5, 1.0, 'disrtibution of price over time')


calendar.describe().price

count    640469.00000
mean        192.45391
std         140.55155
min          11.00000
25%          85.00000
50%         150.00000
75%         250.00000
max         999.00000
Name: price, dtype: float64


calendar_new = calendar[(calendar['price'] >20) & (calendar['price'] <500)]
calendar_new.head()


plt.rcParams["figure.figsize"] = (20,8)
sns.distplot(calendar_new['price'], hist=True, kde=True, 
             bins=int(180/5), color = 'darkblue', 
             hist_kws={'edgecolor':'black'},
             kde_kws={'linewidth': 4})
plt.title ('disrtibution of price over time')

C:\Users\eljaz\anaconda3\lib\site-packages\seaborn\distributions.py:2551: FutureWarning:

`distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `histplot` (an axes-level function for histograms).

Text(0.5, 1.0, 'disrtibution of price over time')


plt.rcParams["figure.figsize"] = (20,8)
sns.boxplot( data = calendar_new  , x='month',  y = 'price' )

<AxesSubplot:xlabel='month', ylabel='price'>


listings = pd.read_csv ('listings.csv')
listings.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 3585 entries, 0 to 3584
Data columns (total 95 columns):
 #   Column                            Non-Null Count  Dtype  
---  ------                            --------------  -----  
 0   id                                3585 non-null   int64  
 1   listing_url                       3585 non-null   object 
 2   scrape_id                         3585 non-null   int64  
 3   last_scraped                      3585 non-null   object 
 4   name                              3585 non-null   object 
 5   summary                           3442 non-null   object 
 6   space                             2528 non-null   object 
 7   description                       3585 non-null   object 
 8   experiences_offered               3585 non-null   object 
 9   neighborhood_overview             2170 non-null   object 
 10  notes                             1610 non-null   object 
 11  transit                           2295 non-null   object 
 12  access                            2096 non-null   object 
 13  interaction                       2031 non-null   object 
 14  house_rules                       2393 non-null   object 
 15  thumbnail_url                     2986 non-null   object 
 16  medium_url                        2986 non-null   object 
 17  picture_url                       3585 non-null   object 
 18  xl_picture_url                    2986 non-null   object 
 19  host_id                           3585 non-null   int64  
 20  host_url                          3585 non-null   object 
 21  host_name                         3585 non-null   object 
 22  host_since                        3585 non-null   object 
 23  host_location                     3574 non-null   object 
 24  host_about                        2276 non-null   object 
 25  host_response_time                3114 non-null   object 
 26  host_response_rate                3114 non-null   object 
 27  host_acceptance_rate              3114 non-null   object 
 28  host_is_superhost                 3585 non-null   object 
 29  host_thumbnail_url                3585 non-null   object 
 30  host_picture_url                  3585 non-null   object 
 31  host_neighbourhood                3246 non-null   object 
 32  host_listings_count               3585 non-null   int64  
 33  host_total_listings_count         3585 non-null   int64  
 34  host_verifications                3585 non-null   object 
 35  host_has_profile_pic              3585 non-null   object 
 36  host_identity_verified            3585 non-null   object 
 37  street                            3585 non-null   object 
 38  neighbourhood                     3042 non-null   object 
 39  neighbourhood_cleansed            3585 non-null   object 
 40  neighbourhood_group_cleansed      0 non-null      float64
 41  city                              3583 non-null   object 
 42  state                             3585 non-null   object 
 43  zipcode                           3547 non-null   object 
 44  market                            3571 non-null   object 
 45  smart_location                    3585 non-null   object 
 46  country_code                      3585 non-null   object 
 47  country                           3585 non-null   object 
 48  latitude                          3585 non-null   float64
 49  longitude                         3585 non-null   float64
 50  is_location_exact                 3585 non-null   object 
 51  property_type                     3582 non-null   object 
 52  room_type                         3585 non-null   object 
 53  accommodates                      3585 non-null   int64  
 54  bathrooms                         3571 non-null   float64
 55  bedrooms                          3575 non-null   float64
 56  beds                              3576 non-null   float64
 57  bed_type                          3585 non-null   object 
 58  amenities                         3585 non-null   object 
 59  square_feet                       56 non-null     float64
 60  price                             3585 non-null   object 
 61  weekly_price                      892 non-null    object 
 62  monthly_price                     888 non-null    object 
 63  security_deposit                  1342 non-null   object 
 64  cleaning_fee                      2478 non-null   object 
 65  guests_included                   3585 non-null   int64  
 66  extra_people                      3585 non-null   object 
 67  minimum_nights                    3585 non-null   int64  
 68  maximum_nights                    3585 non-null   int64  
 69  calendar_updated                  3585 non-null   object 
 70  has_availability                  0 non-null      float64
 71  availability_30                   3585 non-null   int64  
 72  availability_60                   3585 non-null   int64  
 73  availability_90                   3585 non-null   int64  
 74  availability_365                  3585 non-null   int64  
 75  calendar_last_scraped             3585 non-null   object 
 76  number_of_reviews                 3585 non-null   int64  
 77  first_review                      2829 non-null   object 
 78  last_review                       2829 non-null   object 
 79  review_scores_rating              2772 non-null   float64
 80  review_scores_accuracy            2762 non-null   float64
 81  review_scores_cleanliness         2767 non-null   float64
 82  review_scores_checkin             2765 non-null   float64
 83  review_scores_communication       2767 non-null   float64
 84  review_scores_location            2763 non-null   float64
 85  review_scores_value               2764 non-null   float64
 86  requires_license                  3585 non-null   object 
 87  license                           0 non-null      float64
 88  jurisdiction_names                0 non-null      float64
 89  instant_bookable                  3585 non-null   object 
 90  cancellation_policy               3585 non-null   object 
 91  require_guest_profile_picture     3585 non-null   object 
 92  require_guest_phone_verification  3585 non-null   object 
 93  calculated_host_listings_count    3585 non-null   int64  
 94  reviews_per_month                 2829 non-null   float64
dtypes: float64(18), int64(15), object(62)
memory usage: 2.6+ MB


listings.columns

Index(['id', 'listing_url', 'scrape_id', 'last_scraped', 'name', 'summary',
       'space', 'description', 'experiences_offered', 'neighborhood_overview',
       'notes', 'transit', 'access', 'interaction', 'house_rules',
       'thumbnail_url', 'medium_url', 'picture_url', 'xl_picture_url',
       'host_id', 'host_url', 'host_name', 'host_since', 'host_location',
       'host_about', 'host_response_time', 'host_response_rate',
       'host_acceptance_rate', 'host_is_superhost', 'host_thumbnail_url',
       'host_picture_url', 'host_neighbourhood', 'host_listings_count',
       'host_total_listings_count', 'host_verifications',
       'host_has_profile_pic', 'host_identity_verified', 'street',
       'neighbourhood', 'neighbourhood_cleansed',
       'neighbourhood_group_cleansed', 'city', 'state', 'zipcode', 'market',
       'smart_location', 'country_code', 'country', 'latitude', 'longitude',
       'is_location_exact', 'property_type', 'room_type', 'accommodates',
       'bathrooms', 'bedrooms', 'beds', 'bed_type', 'amenities', 'square_feet',
       'price', 'weekly_price', 'monthly_price', 'security_deposit',
       'cleaning_fee', 'guests_included', 'extra_people', 'minimum_nights',
       'maximum_nights', 'calendar_updated', 'has_availability',
       'availability_30', 'availability_60', 'availability_90',
       'availability_365', 'calendar_last_scraped', 'number_of_reviews',
       'first_review', 'last_review', 'review_scores_rating',
       'review_scores_accuracy', 'review_scores_cleanliness',
       'review_scores_checkin', 'review_scores_communication',
       'review_scores_location', 'review_scores_value', 'requires_license',
       'license', 'jurisdiction_names', 'instant_bookable',
       'cancellation_policy', 'require_guest_profile_picture',
       'require_guest_phone_verification', 'calculated_host_listings_count',
       'reviews_per_month'],
      dtype='object')


listings.head()


listings['price']  = listings['price'].str.replace(',', '')
listings['price']  = listings['price'].str.replace('$', '')
listings['price'] = pd.to_numeric(listings['price'])


listings['last_scraped'] = pd.to_datetime(listings.last_scraped)


listings['cleaning_fee']  = listings['cleaning_fee'].str.replace(',', '')
listings['cleaning_fee']  = listings['cleaning_fee'].str.replace('$', '')
listings['cleaning_fee'] = pd.to_numeric(listings['cleaning_fee'])


drop_column = ['id','host_id', 'listing_url' , 'scrape_id','jurisdiction_names','neighbourhood_group_cleansed','has_availability','license','neighbourhood_cleansed','has_availability', 'square_feet',
             'thumbnail_url' , 'medium_url' , 'picture_url' 
             , 'xl_picture_url' , 'host_url', 'host_thumbnail_url', 'host_picture_url']


listings.drop(drop_column , axis = 1  , inplace = True )


listings.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 3585 entries, 0 to 3584
Data columns (total 78 columns):
 #   Column                            Non-Null Count  Dtype         
---  ------                            --------------  -----         
 0   last_scraped                      3585 non-null   datetime64[ns]
 1   name                              3585 non-null   object        
 2   summary                           3442 non-null   object        
 3   space                             2528 non-null   object        
 4   description                       3585 non-null   object        
 5   experiences_offered               3585 non-null   object        
 6   neighborhood_overview             2170 non-null   object        
 7   notes                             1610 non-null   object        
 8   transit                           2295 non-null   object        
 9   access                            2096 non-null   object        
 10  interaction                       2031 non-null   object        
 11  house_rules                       2393 non-null   object        
 12  host_name                         3585 non-null   object        
 13  host_since                        3585 non-null   object        
 14  host_location                     3574 non-null   object        
 15  host_about                        2276 non-null   object        
 16  host_response_time                3114 non-null   object        
 17  host_response_rate                3114 non-null   object        
 18  host_acceptance_rate              3114 non-null   object        
 19  host_is_superhost                 3585 non-null   object        
 20  host_neighbourhood                3246 non-null   object        
 21  host_listings_count               3585 non-null   int64         
 22  host_total_listings_count         3585 non-null   int64         
 23  host_verifications                3585 non-null   object        
 24  host_has_profile_pic              3585 non-null   object        
 25  host_identity_verified            3585 non-null   object        
 26  street                            3585 non-null   object        
 27  neighbourhood                     3042 non-null   object        
 28  city                              3583 non-null   object        
 29  state                             3585 non-null   object        
 30  zipcode                           3547 non-null   object        
 31  market                            3571 non-null   object        
 32  smart_location                    3585 non-null   object        
 33  country_code                      3585 non-null   object        
 34  country                           3585 non-null   object        
 35  latitude                          3585 non-null   float64       
 36  longitude                         3585 non-null   float64       
 37  is_location_exact                 3585 non-null   object        
 38  property_type                     3582 non-null   object        
 39  room_type                         3585 non-null   object        
 40  accommodates                      3585 non-null   int64         
 41  bathrooms                         3571 non-null   float64       
 42  bedrooms                          3575 non-null   float64       
 43  beds                              3576 non-null   float64       
 44  bed_type                          3585 non-null   object        
 45  amenities                         3585 non-null   object        
 46  price                             3585 non-null   float64       
 47  weekly_price                      892 non-null    object        
 48  monthly_price                     888 non-null    object        
 49  security_deposit                  1342 non-null   object        
 50  cleaning_fee                      2478 non-null   float64       
 51  guests_included                   3585 non-null   int64         
 52  extra_people                      3585 non-null   object        
 53  minimum_nights                    3585 non-null   int64         
 54  maximum_nights                    3585 non-null   int64         
 55  calendar_updated                  3585 non-null   object        
 56  availability_30                   3585 non-null   int64         
 57  availability_60                   3585 non-null   int64         
 58  availability_90                   3585 non-null   int64         
 59  availability_365                  3585 non-null   int64         
 60  calendar_last_scraped             3585 non-null   object        
 61  number_of_reviews                 3585 non-null   int64         
 62  first_review                      2829 non-null   object        
 63  last_review                       2829 non-null   object        
 64  review_scores_rating              2772 non-null   float64       
 65  review_scores_accuracy            2762 non-null   float64       
 66  review_scores_cleanliness         2767 non-null   float64       
 67  review_scores_checkin             2765 non-null   float64       
 68  review_scores_communication       2767 non-null   float64       
 69  review_scores_location            2763 non-null   float64       
 70  review_scores_value               2764 non-null   float64       
 71  requires_license                  3585 non-null   object        
 72  instant_bookable                  3585 non-null   object        
 73  cancellation_policy               3585 non-null   object        
 74  require_guest_profile_picture     3585 non-null   object        
 75  require_guest_phone_verification  3585 non-null   object        
 76  calculated_host_listings_count    3585 non-null   int64         
 77  reviews_per_month                 2829 non-null   float64       
dtypes: datetime64[ns](1), float64(15), int64(12), object(50)
memory usage: 2.1+ MB


listings.dropna(how = 'all')


listings.shape

(3585, 78)


listings.property_type.dropna()

0           House
1       Apartment
2       Apartment
3           House
4           House
          ...    
3580    Apartment
3581    Apartment
3582    Apartment
3583    Apartment
3584    Apartment
Name: property_type, Length: 3582, dtype: object


# get categorical data 
listings_categorical =listings.select_dtypes(include=['object'])
listings_categorical.shape

(3585, 50)


# get numerical data 
listings_numerical = listings.select_dtypes(include=['int64','float'])
listings_numerical.head()


listings_numerical = listings_numerical.dropna(how = 'all')
listings_numerical.reset_index(inplace=True)
listings_numerical.head()


n_corr = listings.select_dtypes(include=['int64', 'float64']).corr()
mask = np.zeros_like(n_corr)
mask[np.triu_indices_from(mask)] = True
plt.figure(figsize=(24,12))
plt.title('Heatmap of corr of features')
sns.heatmap(n_corr, mask = mask, vmax=.3, square=True, annot=True, fmt='.2f', cmap='coolwarm')
plt.show()


import plotly.express as px
fig = px.scatter_matrix(listings , dimensions=["price", "bathrooms", "bathrooms", "cleaning_fee", "beds"] , color="room_type")
fig.show()


listings['last_scraped'] = listings['last_scraped'].dt.normalize()


listings['last_scraped']

0      2016-09-07
1      2016-09-07
2      2016-09-07
3      2016-09-07
4      2016-09-07
          ...    
3580   2016-09-07
3581   2016-09-07
3582   2016-09-07
3583   2016-09-07
3584   2016-09-07
Name: last_scraped, Length: 3585, dtype: datetime64[ns]


import plotly.express as px
fig = px.scatter_mapbox(listings, lat="latitude", lon="longitude", color="room_type", size="price",
                  color_continuous_scale=px.colors.cyclical.IceFire, animation_frame="last_scraped" , size_max=15, zoom=10,
                  mapbox_style="carto-positron")
fig.show()


listings.property_type.dropna(how = 'all')

0           House
1       Apartment
2       Apartment
3           House
4           House
          ...    
3580    Apartment
3581    Apartment
3582    Apartment
3583    Apartment
3584    Apartment
Name: property_type, Length: 3582, dtype: object


listings.dropna(subset=['summary'], inplace=True)
listings.shape

(3442, 78)


listings.reset_index(inplace=True)
listings.head()


#listings.reset_index(inplace=True)
listings.head()


summary = ''
for i in range(listings.shape[0]):
    summary = summary + listings['summary'][i]


sns.set(color_codes=True)
wordcloud = WordCloud(width = 800, height = 800, 
                background_color ='white', 
                min_font_size = 10).generate(summary) 
  
# plot the WordCloud image                        
plt.figure(figsize = (8, 8), facecolor = None) 
plt.imshow(wordcloud) 
plt.axis("off") 
plt.tight_layout(pad = 0) 
plt.show()


from nltk.tokenize import word_tokenize
import pandas as pd

import nltk
nltk.download('punkt')

[nltk_data] Downloading package punkt to
[nltk_data]     C:\Users\eljaz\AppData\Roaming\nltk_data...
[nltk_data]   Package punkt is already up-to-date!

True


listings['tokens'] = listings['summary'].apply(word_tokenize)
listings['tokens'].head()

0    [Cozy, ,, sunny, ,, family, home, ., Master, b...
1    [Charming, and, quiet, room, in, a, second, fl...
2    [Come, stay, with, a, friendly, ,, middle-aged...
3    [Come, experience, the, comforts, of, home, aw...
4    [My, comfy, ,, clean, and, relaxing, home, is,...
Name: tokens, dtype: object


all_tokens = []
for i in range(listings.shape[0]):
    all_tokens = all_tokens + listings['tokens'][i]


from nltk.probability import FreqDist
fdist = FreqDist(all_tokens)
fdist

FreqDist({',': 12509, '.': 9555, 'and': 6275, 'the': 5404, 'to': 4412, 'a': 3667, 'in': 3120, 'is': 2633, 'of': 2578, 'Boston': 2379, ...})


tokens1 = [word for word in all_tokens if word.isalnum()]


nltk.download('stopwords')
from nltk.corpus import stopwords
english_stopwords = set(stopwords.words('english'))

[nltk_data] Downloading package stopwords to
[nltk_data]     C:\Users\eljaz\AppData\Roaming\nltk_data...
[nltk_data]   Package stopwords is already up-to-date!


tokens2 = [x for x in tokens1 if x.lower() not in english_stopwords]


tokens2_string = ''
for value in tokens2:
    tokens2_string = tokens2_string + value + ' '


wordcloud = WordCloud(width = 800, height = 800, 
                background_color ='white', colormap='plasma', 
                min_font_size = 10).generate(tokens2_string) 

# plot the WordCloud image                        
plt.figure(figsize = (8, 8), facecolor = None) 
plt.imshow(wordcloud) 
plt.axis("off") 
plt.tight_layout(pad = 0) 
plt.show()


fdist1 = FreqDist(tokens2)
fdist1
counts = pd.Series(fdist1)
counts = counts[:20]
counts.sort_values(ascending=False, inplace=True)
counts

walk          994
bedroom       846
home          519
high          137
family        125
ceilings      124
sunny         113
charming       80
stores         78
garden         70
Cozy           58
Short          34
Master         13
village        11
play           10
Deck            6
attractive      2
hens            1
beehives        1
structure       1
dtype: int64


#Generates bar graph
ax = counts.sort_values(ascending=True).plot(kind='barh', figsize=(10, 10), fontsize=12)

#X axis text and display style of categories
ax.set_xlabel("Frequency", fontsize=12)

#Y axis text
ax.set_ylabel("Word", fontsize=12)

#Title
ax.set_title("Top 20 words with frequency in summary", fontsize=20)

#Annotations
for i in ax.patches:
    # get_width pulls left or right; get_y pushes up or down
    ax.text(i.get_width()+.1, i.get_y()+.31, str(round((i.get_width()), 2)), fontsize=10, color='dimgrey')


from textblob import TextBlob


TextBlob("Today is a great day").sentiment

Sentiment(polarity=0.8, subjectivity=0.75)


TextBlob("Today is not a great day").sentiment

Sentiment(polarity=-0.4, subjectivity=0.75)


def generate_polarity(summary):
    '''Extract polarity score (-1 to +1) for each comment'''
    return TextBlob(summary).sentiment[0]


listings['polarity'] = listings['summary'].apply(generate_polarity)
listings['polarity'].head()

0    0.229375
1    0.203571
2    0.320238
3    0.238131
4    0.128175
Name: polarity, dtype: float64


def generate_subjectivity(summary):
    '''Extract subjectivity score (0 to +1) for each comment'''
    return TextBlob(summary).sentiment[1]


listings['subjectivity'] = listings['summary'].apply(generate_subjectivity)
listings['subjectivity'].head()

0    0.519583
1    0.388095
2    0.561905
3    0.444986
4    0.468254
Name: subjectivity, dtype: float64


num_bins = 50
plt.figure(figsize=(10,6))
n, bins, patches = plt.hist(listings['polarity'], num_bins, facecolor='green', alpha=0.6)
plt.xlabel('Polarity')
plt.ylabel('Count')
plt.title('Histogram of polarity / sentiment score')
plt.show();


positive_sentiment = listings.nlargest(2000, 'polarity')
positive_sentiment = positive_sentiment[['summary', 'polarity','state','city', 'subjectivity']]
positive_sentiment.style.set_properties(subset=['summary'], **{'width': '300px'})


positive_sentiment.state.value_counts()

MA    2000
Name: state, dtype: int64


num_bins = 50
plt.figure(figsize=(10,6))
n, bins, patches = plt.hist(listings['subjectivity'], num_bins, facecolor='green', alpha=0.6)
plt.xlabel('subjectivity')
plt.ylabel('Count')
plt.title('Histogram of polarity / sentiment score')
plt.show();


positive_sentiment['summary']

148     This is an excellent 132 square foot room! It'...
156     Within close walking distance to the subway, r...
1178    This south-end apartment is located 1 block aw...
1727                    Beacon Hill with a Beautiful Cat!
2481    "Perfect home away from home and close to ever...
                              ...                        
1224    Hello! Modern Room in the heart of Boston with...
1724    No Marathon rentals yet. Sunny corner 1 bed fa...
2716    We are conveniently located within miles of Fr...
3174    Convenient 15 min bus Harvard, MIT, 30 min Lon...
3296    Lovely, sunny, private 1BR available in massiv...
Name: summary, Length: 2000, dtype: object


negative_sentiment = listings.nsmallest(2000, 'polarity')
negative_sentiment = negative_sentiment[['summary', 'city','state','name','notes','polarity']]
negative_sentiment.style.set_properties(subset=['summary'], **{'width': '300px'})


negative_sentiment.city.value_counts()

Boston                       1896
Dorchester                     12
Somerville                     12
Cambridge                      11
Roxbury Crossing               11
Brookline                       9
Brighton                        7
Charlestown                     7
Jamaica Plain                   6
Allston                         6
West Roxbury                    3
Roslindale                      3
East Boston                     2
Milton                          1
Jamaica plain                   1
Jamaica Plain, MA               1
ALLSTON                         1
Roslindale, Boston              1
Hyde Park                       1
波士顿                             1
ROXBURY CROSSING                1
Mission Hill, Boston            1
Brighton                        1
Jamaica Plain, Boston           1
Jamaica Plain (Boston)          1
Boston, Massachusetts, US       1
dorchester, boston              1
Name: city, dtype: int64


negative_sentiment[['summary' ,'notes']]


import spacy
nlp = spacy.load('en_core_web_sm')


def generate_named_entities(comment):
    '''Return the text snippet and its corresponding enrity label in a list'''
    return [(ent.text.strip(), ent.label_) for ent in nlp(comment).ents]


listings['named_entities'] = listings['summary'].apply(generate_named_entities)
listings.head()


from spacy import displacy


for i in range(10,40):
    if listings['named_entities'][i]:
        displacy.render(nlp(listings['summary'][i]), style='ent', jupyter=True)


listings_final = listings[(listings['price'] > 20) & (listings['price'] < 500)]
listings_final.info()

<class 'pandas.core.frame.DataFrame'>
Int64Index: 3350 entries, 0 to 3441
Data columns (total 83 columns):
 #   Column                            Non-Null Count  Dtype         
---  ------                            --------------  -----         
 0   index                             3350 non-null   int64         
 1   last_scraped                      3350 non-null   datetime64[ns]
 2   name                              3350 non-null   object        
 3   summary                           3350 non-null   object        
 4   space                             2321 non-null   object        
 5   description                       3350 non-null   object        
 6   experiences_offered               3350 non-null   object        
 7   neighborhood_overview             2079 non-null   object        
 8   notes                             1551 non-null   object        
 9   transit                           2203 non-null   object        
 10  access                            2015 non-null   object        
 11  interaction                       1951 non-null   object        
 12  house_rules                       2224 non-null   object        
 13  host_name                         3350 non-null   object        
 14  host_since                        3350 non-null   object        
 15  host_location                     3339 non-null   object        
 16  host_about                        2082 non-null   object        
 17  host_response_time                2905 non-null   object        
 18  host_response_rate                2905 non-null   object        
 19  host_acceptance_rate              2905 non-null   object        
 20  host_is_superhost                 3350 non-null   object        
 21  host_neighbourhood                3022 non-null   object        
 22  host_listings_count               3350 non-null   int64         
 23  host_total_listings_count         3350 non-null   int64         
 24  host_verifications                3350 non-null   object        
 25  host_has_profile_pic              3350 non-null   object        
 26  host_identity_verified            3350 non-null   object        
 27  street                            3350 non-null   object        
 28  neighbourhood                     2820 non-null   object        
 29  city                              3348 non-null   object        
 30  state                             3350 non-null   object        
 31  zipcode                           3313 non-null   object        
 32  market                            3336 non-null   object        
 33  smart_location                    3350 non-null   object        
 34  country_code                      3350 non-null   object        
 35  country                           3350 non-null   object        
 36  latitude                          3350 non-null   float64       
 37  longitude                         3350 non-null   float64       
 38  is_location_exact                 3350 non-null   object        
 39  property_type                     3347 non-null   object        
 40  room_type                         3350 non-null   object        
 41  accommodates                      3350 non-null   int64         
 42  bathrooms                         3345 non-null   float64       
 43  bedrooms                          3348 non-null   float64       
 44  beds                              3342 non-null   float64       
 45  bed_type                          3350 non-null   object        
 46  amenities                         3350 non-null   object        
 47  price                             3350 non-null   float64       
 48  weekly_price                      827 non-null    object        
 49  monthly_price                     783 non-null    object        
 50  security_deposit                  1254 non-null   object        
 51  cleaning_fee                      2329 non-null   float64       
 52  guests_included                   3350 non-null   int64         
 53  extra_people                      3350 non-null   object        
 54  minimum_nights                    3350 non-null   int64         
 55  maximum_nights                    3350 non-null   int64         
 56  calendar_updated                  3350 non-null   object        
 57  availability_30                   3350 non-null   int64         
 58  availability_60                   3350 non-null   int64         
 59  availability_90                   3350 non-null   int64         
 60  availability_365                  3350 non-null   int64         
 61  calendar_last_scraped             3350 non-null   object        
 62  number_of_reviews                 3350 non-null   int64         
 63  first_review                      2638 non-null   object        
 64  last_review                       2638 non-null   object        
 65  review_scores_rating              2586 non-null   float64       
 66  review_scores_accuracy            2577 non-null   float64       
 67  review_scores_cleanliness         2582 non-null   float64       
 68  review_scores_checkin             2580 non-null   float64       
 69  review_scores_communication       2582 non-null   float64       
 70  review_scores_location            2578 non-null   float64       
 71  review_scores_value               2579 non-null   float64       
 72  requires_license                  3350 non-null   object        
 73  instant_bookable                  3350 non-null   object        
 74  cancellation_policy               3350 non-null   object        
 75  require_guest_profile_picture     3350 non-null   object        
 76  require_guest_phone_verification  3350 non-null   object        
 77  calculated_host_listings_count    3350 non-null   int64         
 78  reviews_per_month                 2638 non-null   float64       
 79  tokens                            3350 non-null   object        
 80  polarity                          3350 non-null   float64       
 81  subjectivity                      3350 non-null   float64       
 82  named_entities                    3350 non-null   object        
dtypes: datetime64[ns](1), float64(17), int64(13), object(52)
memory usage: 2.1+ MB


# select numeric cols
num_cols = ['price', 'latitude','longitude', 'accommodates', 'bedrooms', 'bathrooms', 'beds', 
             'cleaning_fee', 'guests_included', 'availability_30', 'availability_60', 'availability_90', 
             'availability_365', 'review_scores_rating', 'review_scores_accuracy', 'review_scores_cleanliness', 
             'review_scores_location', 'review_scores_value', 'calculated_host_listings_count']

numeric = listings_final.select_dtypes(include=['int64', 'float64'])[num_cols]
print(numeric.info())

# transform categorical columns into numeric and prepare new data frame
cat_cols = ['host_response_time', 'host_is_superhost', 'room_type', 'bed_type',
             'cancellation_policy', 'property_type', 'host_identity_verified', 'instant_bookable',
            'host_has_profile_pic', 'require_guest_profile_picture', 'require_guest_phone_verification']

numeric[cat_cols] = listings_final[cat_cols]

num_copy = numeric.copy()

num_copy = num_copy.replace({ "host_is_superhost": {"t": 1, "f": 2}, "instant_bookable": {"t": 1, "f": 2}, 
                                "host_identity_verified": {"t": 1, "f": 2}, "require_guest_profile_picture": {"t": 1, "f": 2},
                                "room_type": {"Entire home/apt": 1, "Private room": 2, "Shared room": 3}, "host_has_profile_pic": {"t": 1, "f": 2},
                               "bed_type": {"Real Bed": 1, "Futon": 2, "Airbed": 3, "Pull-out Sofa": 4, "Couch": 5},
                               "require_guest_phone_verification": {"t": 1, "f": 2},
                               "cancellation_policy": {"moderate": 1, "flexible": 2, "strict": 3, "super_strict_30": 4}})

dummies = pd.get_dummies(num_copy)
print(dummies.info())

<class 'pandas.core.frame.DataFrame'>
Int64Index: 3350 entries, 0 to 3441
Data columns (total 19 columns):
 #   Column                          Non-Null Count  Dtype  
---  ------                          --------------  -----  
 0   price                           3350 non-null   float64
 1   latitude                        3350 non-null   float64
 2   longitude                       3350 non-null   float64
 3   accommodates                    3350 non-null   int64  
 4   bedrooms                        3348 non-null   float64
 5   bathrooms                       3345 non-null   float64
 6   beds                            3342 non-null   float64
 7   cleaning_fee                    2329 non-null   float64
 8   guests_included                 3350 non-null   int64  
 9   availability_30                 3350 non-null   int64  
 10  availability_60                 3350 non-null   int64  
 11  availability_90                 3350 non-null   int64  
 12  availability_365                3350 non-null   int64  
 13  review_scores_rating            2586 non-null   float64
 14  review_scores_accuracy          2577 non-null   float64
 15  review_scores_cleanliness       2582 non-null   float64
 16  review_scores_location          2578 non-null   float64
 17  review_scores_value             2579 non-null   float64
 18  calculated_host_listings_count  3350 non-null   int64  
dtypes: float64(12), int64(7)
memory usage: 523.4 KB
None
<class 'pandas.core.frame.DataFrame'>
Int64Index: 3350 entries, 0 to 3441
Data columns (total 45 columns):
 #   Column                                 Non-Null Count  Dtype  
---  ------                                 --------------  -----  
 0   price                                  3350 non-null   float64
 1   latitude                               3350 non-null   float64
 2   longitude                              3350 non-null   float64
 3   accommodates                           3350 non-null   int64  
 4   bedrooms                               3348 non-null   float64
 5   bathrooms                              3345 non-null   float64
 6   beds                                   3342 non-null   float64
 7   cleaning_fee                           2329 non-null   float64
 8   guests_included                        3350 non-null   int64  
 9   availability_30                        3350 non-null   int64  
 10  availability_60                        3350 non-null   int64  
 11  availability_90                        3350 non-null   int64  
 12  availability_365                       3350 non-null   int64  
 13  review_scores_rating                   2586 non-null   float64
 14  review_scores_accuracy                 2577 non-null   float64
 15  review_scores_cleanliness              2582 non-null   float64
 16  review_scores_location                 2578 non-null   float64
 17  review_scores_value                    2579 non-null   float64
 18  calculated_host_listings_count         3350 non-null   int64  
 19  host_is_superhost                      3350 non-null   int64  
 20  room_type                              3350 non-null   int64  
 21  bed_type                               3350 non-null   int64  
 22  cancellation_policy                    3350 non-null   int64  
 23  host_identity_verified                 3350 non-null   int64  
 24  instant_bookable                       3350 non-null   int64  
 25  host_has_profile_pic                   3350 non-null   int64  
 26  require_guest_profile_picture          3350 non-null   int64  
 27  require_guest_phone_verification       3350 non-null   int64  
 28  host_response_time_a few days or more  3350 non-null   uint8  
 29  host_response_time_within a day        3350 non-null   uint8  
 30  host_response_time_within a few hours  3350 non-null   uint8  
 31  host_response_time_within an hour      3350 non-null   uint8  
 32  property_type_Apartment                3350 non-null   uint8  
 33  property_type_Bed & Breakfast          3350 non-null   uint8  
 34  property_type_Boat                     3350 non-null   uint8  
 35  property_type_Camper/RV                3350 non-null   uint8  
 36  property_type_Condominium              3350 non-null   uint8  
 37  property_type_Dorm                     3350 non-null   uint8  
 38  property_type_Entire Floor             3350 non-null   uint8  
 39  property_type_Guesthouse               3350 non-null   uint8  
 40  property_type_House                    3350 non-null   uint8  
 41  property_type_Loft                     3350 non-null   uint8  
 42  property_type_Other                    3350 non-null   uint8  
 43  property_type_Townhouse                3350 non-null   uint8  
 44  property_type_Villa                    3350 non-null   uint8  
dtypes: float64(12), int64(16), uint8(17)
memory usage: 814.6 KB
None


for col in dummies:
    dummies[col].fillna((dummies[col].mean()), inplace=True)


y = dummies['price'].astype(float)


X = dummies.drop('price',  axis =1 )


X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.30, random_state=42)


lm_model =  LinearRegression()
lm_model.fit(X_train , y_train )
y_test_preds = lm_model.predict(X_test)
y_train_preds = lm_model.predict(X_train)


#r2 value
r2_scores_test = r2_score(y_test, y_test_preds)
r2_scores_train = r2_score(y_train, y_train_preds)


print (r2_scores_test , r2_scores_train )

0.5886105866343003 0.5967662049596525


fig = plt.figure(figsize =(10, 4))
fig.subplots_adjust(hspace=0.4, wspace=0.4)
ax = plt.axes(aspect = 'equal')
plt.subplot(121)
plt.title('regplot for distribution', fontsize=14)
sns.regplot(y_test, y_test_preds, color='blue')
plt.xlabel('Real Price', fontsize=14)
plt.ylabel('Predicted Price', fontsize=14)
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)

plt.subplot(122)
sns.distplot(y_test_preds, hist=False,
             kde_kws={'color': 'b', 'lw': 2, 'label': 'Predicted price'})
sns.distplot(y_test, hist=False,
             kde_kws={'color': 'r', 'lw': 2, 'label': 'Real price'})
plt.title('Distribution comparison', fontsize=14)
plt.ylabel('Probablity', fontsize=12)
plt.xlabel('Price (USD)', fontsize=12)
plt.xticks(fontsize=12)
plt.yticks(fontsize=12)
plt.legend(['Predicted price', 'Real price'], prop={"size":12})
plt.show()

C:\Users\eljaz\anaconda3\lib\site-packages\seaborn\_decorators.py:36: FutureWarning:

Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.

C:\Users\eljaz\anaconda3\lib\site-packages\seaborn\distributions.py:2551: FutureWarning:

`distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `kdeplot` (an axes-level function for kernel density plots).

C:\Users\eljaz\anaconda3\lib\site-packages\seaborn\distributions.py:2551: FutureWarning:

`distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `kdeplot` (an axes-level function for kernel density plots).


feature_importance = pd.DataFrame(
    {'features': X.columns, 'coefficients': lm_model.coef_}
).sort_values(by='coefficients')
feature_importance['features'] = feature_importance['features']

import plotly.express as px
fig = px.bar(x='features', y='coefficients',
             data_frame=feature_importance, height=60)
fig.show();


from sklearn.ensemble import RandomForestRegressor
# Create instance of Random Forest Regressor and evaluate model
model_rf = RandomForestRegressor(n_estimators=76, random_state=42) 
model_rf.fit(X_train , y_train )
y_test_preds = model_rf.predict(X_test)
y_train_preds = model_rf.predict(X_train)
r2_scores_test = r2_score(y_test, y_test_preds)
r2_scores_train = r2_score(y_train, y_train_preds)
print (r2_scores_test , r2_scores_train )

0.7001342745510983 0.9555994811354324


fig = plt.figure(figsize =(10, 4))
fig.subplots_adjust(hspace=0.4, wspace=0.4)
ax = plt.axes(aspect = 'equal')
plt.subplot(121)
plt.title('regplot for distribution', fontsize=14)
sns.regplot(y_test, y_test_preds, color='blue')
plt.xlabel('Real Price', fontsize=14)
plt.ylabel('Predicted Price', fontsize=14)
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)

plt.subplot(122)
sns.distplot(y_test_preds, hist=False,
             kde_kws={'color': 'b', 'lw': 2, 'label': 'Predicted price'})
sns.distplot(y_test, hist=False,
             kde_kws={'color': 'r', 'lw': 2, 'label': 'Real price'})
plt.title('Distribution comparison', fontsize=14)
plt.ylabel('Probablity', fontsize=12)
plt.xlabel('Price (USD)', fontsize=12)
plt.xticks(fontsize=12)
plt.yticks(fontsize=12)
plt.legend(['Predicted price', 'Real price'], prop={"size":12})
plt.show()

C:\Users\eljaz\anaconda3\lib\site-packages\seaborn\_decorators.py:36: FutureWarning:

Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.

C:\Users\eljaz\anaconda3\lib\site-packages\seaborn\distributions.py:2551: FutureWarning:

`distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `kdeplot` (an axes-level function for kernel density plots).

C:\Users\eljaz\anaconda3\lib\site-packages\seaborn\distributions.py:2551: FutureWarning:

`distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `kdeplot` (an axes-level function for kernel density plots).


<module 'pandas' from '/usr/local/lib/python3.6/site-packages/pandas/__init__.py'>
globals()['np']
<module 'numpy' from '/usr/local/lib/python3.6/site-packages/numpy/__init__.py'>

	listing_id	date	available	price
365	3075044	2017-08-22	t	65.0
366	3075044	2017-08-21	t	65.0
367	3075044	2017-08-20	t	65.0
368	3075044	2017-08-19	t	75.0
369	3075044	2017-08-18	t	75.0
370	3075044	2017-08-17	t	65.0
371	3075044	2017-08-16	t	65.0
372	3075044	2017-08-15	t	65.0
373	3075044	2017-08-14	t	65.0
374	3075044	2017-08-13	t	65.0
375	3075044	2017-08-12	t	75.0
376	3075044	2017-08-11	t	75.0
377	3075044	2017-08-10	t	65.0
378	3075044	2017-08-09	t	65.0
379	3075044	2017-08-08	t	65.0
380	3075044	2017-08-07	t	65.0
381	3075044	2017-08-06	t	65.0
382	3075044	2017-08-05	t	75.0
383	3075044	2017-08-04	t	75.0
384	3075044	2017-08-03	t	65.0

	listing_id	date	available	price	quarter	month
365	3075044	2017-08-22	t	65.0	2017Q3	2017-08
366	3075044	2017-08-21	t	65.0	2017Q3	2017-08
367	3075044	2017-08-20	t	65.0	2017Q3	2017-08
368	3075044	2017-08-19	t	75.0	2017Q3	2017-08
369	3075044	2017-08-18	t	75.0	2017Q3	2017-08

	id	listing_url	scrape_id	last_scraped	name	summary	space	description	experiences_offered	neighborhood_overview	...	review_scores_value	requires_license	license	jurisdiction_names	instant_bookable	cancellation_policy	require_guest_profile_picture	require_guest_phone_verification	calculated_host_listings_count	reviews_per_month
0	12147973	https://www.airbnb.com/rooms/12147973	20160906204935	2016-09-07	Sunny Bungalow in the City	Cozy, sunny, family home. Master bedroom high...	The house has an open and cozy feel at the sam...	Cozy, sunny, family home. Master bedroom high...	none	Roslindale is quiet, convenient and friendly. ...	...	NaN	f	NaN	NaN	f	moderate	f	f	1	NaN
1	3075044	https://www.airbnb.com/rooms/3075044	20160906204935	2016-09-07	Charming room in pet friendly apt	Charming and quiet room in a second floor 1910...	Small but cozy and quite room with a full size...	Charming and quiet room in a second floor 1910...	none	The room is in Roslindale, a diverse and prima...	...	9.0	f	NaN	NaN	t	moderate	f	f	1	1.30
2	6976	https://www.airbnb.com/rooms/6976	20160906204935	2016-09-07	Mexican Folk Art Haven in Boston	Come stay with a friendly, middle-aged guy in ...	Come stay with a friendly, middle-aged guy in ...	Come stay with a friendly, middle-aged guy in ...	none	The LOCATION: Roslindale is a safe and diverse...	...	10.0	f	NaN	NaN	f	moderate	t	f	1	0.47
3	1436513	https://www.airbnb.com/rooms/1436513	20160906204935	2016-09-07	Spacious Sunny Bedroom Suite in Historic Home	Come experience the comforts of home away from...	Most places you find in Boston are small howev...	Come experience the comforts of home away from...	none	Roslindale is a lovely little neighborhood loc...	...	10.0	f	NaN	NaN	f	moderate	f	f	1	1.00
4	7651065	https://www.airbnb.com/rooms/7651065	20160906204935	2016-09-07	Come Home to Boston	My comfy, clean and relaxing home is one block...	Clean, attractive, private room, one block fro...	My comfy, clean and relaxing home is one block...	none	I love the proximity to downtown, the neighbor...	...	10.0	f	NaN	NaN	f	flexible	f	f	1	2.25

	last_scraped	name	summary	space	description	experiences_offered	neighborhood_overview	notes	transit	access	...	review_scores_communication	review_scores_location	review_scores_value	requires_license	instant_bookable	cancellation_policy	require_guest_profile_picture	require_guest_phone_verification	calculated_host_listings_count	reviews_per_month
0	2016-09-07	Sunny Bungalow in the City	Cozy, sunny, family home. Master bedroom high...	The house has an open and cozy feel at the sam...	Cozy, sunny, family home. Master bedroom high...	none	Roslindale is quiet, convenient and friendly. ...	NaN	The bus stop is 2 blocks away, and frequent. B...	You will have access to 2 bedrooms, a living r...	...	NaN	NaN	NaN	f	f	moderate	f	f	1	NaN
1	2016-09-07	Charming room in pet friendly apt	Charming and quiet room in a second floor 1910...	Small but cozy and quite room with a full size...	Charming and quiet room in a second floor 1910...	none	The room is in Roslindale, a diverse and prima...	If you don't have a US cell phone, you can tex...	Plenty of safe street parking. Bus stops a few...	Apt has one more bedroom (which I use) and lar...	...	10.0	9.0	9.0	f	t	moderate	f	f	1	1.30
2	2016-09-07	Mexican Folk Art Haven in Boston	Come stay with a friendly, middle-aged guy in ...	Come stay with a friendly, middle-aged guy in ...	Come stay with a friendly, middle-aged guy in ...	none	The LOCATION: Roslindale is a safe and diverse...	I am in a scenic part of Boston with a couple ...	PUBLIC TRANSPORTATION: From the house, quick p...	I am living in the apartment during your stay,...	...	10.0	9.0	10.0	f	f	moderate	t	f	1	0.47
3	2016-09-07	Spacious Sunny Bedroom Suite in Historic Home	Come experience the comforts of home away from...	Most places you find in Boston are small howev...	Come experience the comforts of home away from...	none	Roslindale is a lovely little neighborhood loc...	Please be mindful of the property as it is old...	There are buses that stop right in front of th...	The basement has a washer dryer and gym area. ...	...	10.0	10.0	10.0	f	f	moderate	f	f	1	1.00
4	2016-09-07	Come Home to Boston	My comfy, clean and relaxing home is one block...	Clean, attractive, private room, one block fro...	My comfy, clean and relaxing home is one block...	none	I love the proximity to downtown, the neighbor...	I have one roommate who lives on the lower lev...	From Logan Airport and South Station you have...	You will have access to the front and side por...	...	10.0	9.0	10.0	f	f	flexible	f	f	1	2.25
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
3580	2016-09-07	Big cozy room near T	5 min walking to Orange Line subway with 2 sto...	NaN	5 min walking to Orange Line subway with 2 sto...	none	NaN	NaN	NaN	NaN	...	10.0	8.0	9.0	f	t	strict	f	f	8	0.34
3581	2016-09-07	BU Apartment DexterPark Bright room	Most popular apartment in BU, best located in ...	Best location in BU	Most popular apartment in BU, best located in ...	none	NaN	NaN	There is green line, BU shuttle in front of th...	NaN	...	NaN	NaN	NaN	f	f	strict	f	f	2	NaN
3582	2016-09-07	Gorgeous funky apartment	Funky little apartment close to public transpo...	Modern and relaxed space with many facilities ...	Funky little apartment close to public transpo...	none	Cambridge is a short walk into Boston, and set...	Depending on when you arrive, I can be here to...	Public transport is 5 minuts away, but walking...	The whole place including social areas is your...	...	NaN	NaN	NaN	f	f	flexible	f	f	1	NaN
3583	2016-09-07	Great Location; Train and Restaurants	My place is close to Taco Loco Mexican Grill, ...	NaN	My place is close to Taco Loco Mexican Grill, ...	none	NaN	NaN	NaN	NaN	...	9.0	8.0	7.0	f	f	strict	f	f	1	2.00
3584	2016-09-07	(K1) Private Room near Harvard/MIT	My place is close to My home is a warm and fri...	To ensure a smooth check in: 1. You MUST have ...	My place is close to My home is a warm and fri...	none	NaN	NaN	NaN	NaN	...	NaN	NaN	NaN	f	t	flexible	f	f	3	NaN

	host_listings_count	host_total_listings_count	latitude	longitude	accommodates	bathrooms	bedrooms	beds	price	cleaning_fee	...	number_of_reviews	review_scores_rating	review_scores_accuracy	review_scores_cleanliness	review_scores_checkin	review_scores_communication	review_scores_location	review_scores_value	calculated_host_listings_count	reviews_per_month
0	1	1	42.282619	-71.133068	4	1.5	2.0	3.0	250.0	35.0	...	0	NaN	NaN	NaN	NaN	NaN	NaN	NaN	1	NaN
1	1	1	42.286241	-71.134374	2	1.0	1.0	1.0	65.0	10.0	...	36	94.0	10.0	9.0	10.0	10.0	9.0	9.0	1	1.30
2	1	1	42.292438	-71.135765	2	1.0	1.0	1.0	65.0	NaN	...	41	98.0	10.0	9.0	10.0	10.0	9.0	10.0	1	0.47
3	1	1	42.281106	-71.121021	4	1.0	1.0	2.0	75.0	50.0	...	1	100.0	10.0	10.0	10.0	10.0	10.0	10.0	1	1.00
4	1	1	42.284512	-71.136258	2	1.5	1.0	2.0	79.0	15.0	...	29	99.0	10.0	10.0	10.0	10.0	9.0	10.0	1	2.25

CRISP-DM Process¶

I- Business understanding¶

so we try to figure out the below issues

I.I question neeed to ansewrs

Rigth skewed¶

Removing Outlier¶

price of major unit lies between 20 to 500 and others is outliers¶

After Removing Of Outlier¶

Listings¶

studing the correlation between numerical parameters¶

2.1 analysis of summary column by tools on NLP by measuring¶

3-prediction of price according to different parameter¶

3.1 linear Regression¶

Final Outcomes¶

	summary	notes
1970	6th floor overlooking the Boston Common and th...	Great location! The Red Park Street Train sto...
1364	One bedroom apartment on Beacon St. Walking di...	NaN
1998	This Sunny Theater District apartment features...	NaN
2470	Furnished 1 BR, 1 Bath (approx. 585 sq. feet) ...	NaN
418	The apartment is a minute away from subway and...	NaN
...	...	...
29	My place is good for couples, solo adventurers...	NaN
1568	My place is close to Little Asia, Orient Heigh...	Laundry is available near the apartment. The m...
168	Quiet, peaceful room with pvt bath on your own...	While you don't need a car if you stay with us...
1630	Feel at home in this recently renovated 1 br /...	NaN
3193	The Apartment is very clean, nicely decorated ...	NaN

	summary	polarity	state	city	subjectivity
148	This is an excellent 132 square foot room! It's a 10 minute walk to the T. 5 minute walk to the Arboretum, about 7 minutes to the pond. Room has internet, kitchen access (fridge, freezer, stove, microwave, etc..). Please contact me with questions!	1.000000	MA	Boston	1.000000
156	Within close walking distance to the subway, restaurants, gourmet coffee shop, Jamaica Pond this location can't be beat in one of Boston's greatest neighborhoods!	1.000000	MA	Boston	1.000000
1178	This south-end apartment is located 1 block away from Boston's best restaurants on Tremont street. If you don't want to go out, dine one the porch with beautiful views. Fold out couch allows the place to accommodate 4 people. Washer and Dryer too!	1.000000	MA	Boston	0.650000
1727	Beacon Hill with a Beautiful Cat!	1.000000	MA	Boston	1.000000
2481	"Perfect home away from home and close to everything Boston has to offer!"	1.000000	MA	Boston	1.000000
2527	This two bedroom apartment in Boston is perfect for a college visits, friends renting a place for the weekend, and anyone else! This comfy two bed apartment is the perfect alternative to staying at a hotel. AND it comes with parking!	1.000000	MA	Boston	1.000000
3158	This is the best room in Boston!	1.000000	MA	Boston	0.300000
2671	A beautiful Victorian 3 bedroom apartment in Dorchester. The home is a three minute walk to the commuter rail and red/orange bus lines. The home is a 15 minute ride to South Station/ Downtown Boston. Off st. parking. Walking distance to Franklin Park Zoo. Perfect for families!	0.925000	MA	Boston	1.000000
2895	A beautiful Victorian apartment in Dorchester. The home is a three minute walk to the commuter rail and red/orange bus lines. The home is a 15 minute ride to South Station/ Downtown Boston. Off st. parking. Walking distance to Franklin Park Zoo. Perfect for families!	0.925000	MA	Boston	1.000000
2983	Beautiful One-Bedroom Studio in the heart of South Boston blocks away from the beach, bar/restaurant destinations, and minutes from Downtown Boston. A perfect weekend getaway for two!	0.925000	MA	Boston	1.000000
812	The apartment is a perfect location - just a 2 minute walk from the heart of downtown. Seconds from the No. 1 bus line that will take you to Cambridge in ~10 minutes and the subways lines that will take you virtually everywhere. Area is very safe!	0.906250	MA	Boston	0.825000
741	Come stay at a great location in Boston! Walking distance to everything the city has to offer! This two bedroom one bath apartment with exposed brick located on the 4th floor is a great home away from home.	0.900000	MA	Boston	0.750000
3151	This is a good room!	0.875000	MA	Boston	0.600000
1601	Come stay in our beautiful studio, close to everything Boston has to offer.	0.850000	MA	Boston	1.000000
1750	Beautiful Studio offering the luxury of home furnishings with the accessibility of downtown Boston and Walking distance to MGH.	0.850000	MA	Boston	1.000000
1835	Beautiful Studio offering the luxury of home furnishings with the accessibility of downtown Boston and Walking distance to MGH.	0.850000	MA	Boston	1.000000
1973	Beautifully renovated and exclusive furnished apartment in the heart of Boston's Downtown Crossing. 8 Winter Street is fully furnished and equipped with flexible lease terms. This apartment is close to transportation and convenient to shops and restaurants.	0.850000	MA	Boston	1.000000
2029	Beautifully renovated and exclusive furnished apartment in the heart of Boston's Downtown Crossing. On Winter Street, this apartment is fully furnished and equipped with flexible lease terms. Close to transportation and convenient to shops and restaurants.	0.850000	MA	Boston	1.000000
2036	Beautifully renovated and exclusive furnished apartment in the heart of Boston's Downtown Crossing. Fully furnished and equipped with flexible lease terms. 8 Winter Street is close to transportation and convenient to shops and restaurants.	0.850000	MA	Boston	1.000000
2920	From exquisite apartments to alluring amenities, Waterside Place gives you the perfect space to make the most of Boston living.	0.833333	MA	Boston	0.833333
2636	Excellent 1 bedroom apartment, very clean, ideally located in the heart of UMASS Boston. It has everything you need!	0.825556	MA	Boston	0.970000
648	My place is good for couples, solo adventurers, and business travelers. You will get a great roof top view of the city and even the fireworks for the 4th. This unit is in the heart of the north end. BOOK IT NOW!!!	0.825521	MA	Boston	0.616667
3099	A beautiful up-to date Condo located 1 block from Andrew Station, walking distance to M Street Beach, the park and popular restaurants like Lincoln, Loco and Capo! Great neighborhood, location, layout.	0.800000	MA	Boston	0.883333
2656	My place is great for whoever visiting Boston	0.800000	MA	Dorchester	0.750000
3155	Great location. Queen bed	0.800000	MA	Boston	0.750000
1053	Hey Travelers! Christina and I are happy to welcome to you to our home! Drew has lived here for over 7 years & loves welcoming guests from all over the world. We share our guest room with all of you that is; fully furnished, warm, and welcoming.	0.800000	MA	Boston	0.833333
1179	Perfect location on Newbury Street !! Ideal for visiting, shopping and enjoying Boston	0.800000	MA	Boston	0.866667
2692	Our home is great for students and visitors to Boston. It is about a 5 minute walk to the Morton St Commuter Rail Station that will get you to Downtown Boston in 18 minutes! The property is easily accessible to I-93. Welcome to the Lavender Room!	0.791667	MA	Boston	0.675000
2813	Our home is great for students and visitors to Boston. It is about a 5 minute walk to the Morton St Commuter Rail Station that will get you to Downtown Boston in 18 minutes! The property is easily accessible to I-93. Welcome to the Aqua Room!	0.791667	MA	Boston	0.675000
136	Beautiful 2BD pond-side condo with all amenities. Steps from Jamaica Pond, Centre Street, the Orange line and bus routes to downtown, the location could not be more perfect for a relaxing vacation getaway.	0.783333	MA	Boston	0.833333
2048	Beautifully renovated and exclusive furnished apartment in the heart of Boston's Downtown Crossing. 8 Winter Street is fully furnished and equipped with flexible lease terms. Close to transportation and convenient to shops and restaurants. My place is good for couples, solo adventurers, business travelers, families (with kids), and furry friends (pets).	0.775000	MA	Boston	0.800000
1125	Situated on Dartmouth St in the heart of Boston's South End this Charming 640 sqft 1 bedroom unit is perfect. This well appointed and laid out condo is an amazing place to use as your home while enjoining everything that Boston has to offer.	0.766667	MA	Boston	0.966667
417	This is a beautiful, high-ceilinged bedroom right by the T, with plenty of space and amenities! My husband and I will give you great tips on touring Boston but still leave you plenty of privacy. Peace, comfort, and convenience, all at a great price!	0.751786	MA	Boston	0.758929
1163	Awesome studio all to yourself. Queen sized bed, coffee gadgets galore, TV, and free parking!	0.750000	MA	Boston	0.900000
1412	Perfect size queen bedroom, with an open kitchen concept, living room with a fold out couch, cable tv and wifi and fully stocked kitchen with all towels and linens included. Excellent storage for clothes. Great location!	0.750000	MA	Boston	0.812500
3314	With space to sleep six, this three bedroom in the Allston neighborhood is the perfect launch pad to set off discovering the Greater Boston Area.	0.750000	MA	Boston	0.750000
1526	The apartment is beautiful and extremely clean with amazing location! I also cook Delicious Organic Breakfasts for my guests :) There are two parks within 5 min walking distance with the best view to Boston waterfront. Subway, water transportation and airport nearby. One stop to Boston downtown!	0.744444	MA	Boston	0.816667
1039	My place is walking distance to dozens of some of Boston's best restaurants located here in South End. You’ll love my place because of the location, the cleanliness. It's also good for solo adventurers and business travelers.	0.733333	MA	Boston	0.500000
677	Beautiful apartment, updated and secure apartment in the heart of the city. Come enjoy being within walking distance of all the best that Boston has to offer in the best neighborhood in this world class city!	0.730000	MA	Boston	0.540000
1150	This is a perfectly maintained, updated Condo in the Best Neighborhood in Boston. We have 2 bedrooms and 2 bathrooms, each of the beds has a new queen size mattress. The home has a beautiful 20'x30' deck with city views. You will love it!	0.722273	MA	Boston	0.670909

	summary	city	state	name	notes	polarity
1970	6th floor overlooking the Boston Common and the State House! Sunny, spacious 1 bed in a concierge building with elevator, common roofdeck, laundry on each floor and conveniently located close to everything.	Boston	MA	Bookcase 1bd facing Boston Common	Great location! The Red Park Street Train stop is a 5 minute walk, tour buses stop across the street (great way to learn about our city), restaurants, shopping, banks, etc all within walking distance. And of course the fantastic Boston Common and the Public Gardens are across the street.	-0.337500
1364	One bedroom apartment on Beacon St. Walking distance to Fenway Park, MIT, Newbury and Bolyston Street, the Boston Common, etc. Bus stops in front of the building, nearest T-stop is four blocks away.	Boston	MA	Back Bay Apartment	nan	-0.300000
1998	This Sunny Theater District apartment features an open-concept layout, fully-equipped kitchen and expansive windows! Close to Boston Common and Chinatown	Boston	MA	Sunny 1BR in the Theater District	nan	-0.300000
2470	Furnished 1 BR, 1 Bath (approx. 585 sq. feet) with 1 assigned off-street parking. Full-size bed and queen size murphy bed. Ample closet space. Refrigerator, microwave, and toaster in kitchen. Common laundry in basement.	Boston	MA	Cozy 1 BR APT with assigned parking	nan	-0.300000
418	The apartment is a minute away from subway and bus stops. A small balcony. Wifi and cable close to all tourist attractions, downtown , schools and hospitals	Boston	MA	Apt with walking distance to subway	nan	-0.250000
1786	Address is located on the prestigious "south slope" of Beacon Hill, alongside the Charles River. Short walk to all attractions! Use of kitchen & dining room. Two small non allergenic "hair" dogs, no fur, no dander, no shedding. Internet service: (URL HIDDEN) $2.95 hourly pass* $7.95 daily pass $19.95 weekly pass $54.95 monthly pass	Boston	MA	Beacon Hill, Guestroom with private bath	Parking in Boston is a real challenge. Internet: (URL HIDDEN) $2.95 hourly pass* $7.95 daily pass $19.95 weekly pass $54.95 monthly pass	-0.233333
3250	Our least expensive room - this room has one single bed for one person with bath in the corridor. Includes parking WI-FI and local telephone service.	Boston	MA	Budget Single - share bath	nan	-0.217857
3196	A well-furnished room in a cozy apartment! You'll be surrounded by typical Boston bricks and fall leaves. The apartment is a 1 minute walk from the MBTA B Line, a 25 minute ride downtown.	Boston	MA	Cozy Room in Brookline Apartment	nan	-0.208333
334	Sunny room in apartment conveniently located in green JP: a 5 minute walk to the Stony Brook T stop, and 15 minutes ride downtown, and an abundance of shops, cafes, restaurants, moments away, together with Jamaica Pond and the Arnold Arboretum.	Boston	MA	Bed, Bath and Beyond II in JP	Breakfast foods are provided -- tea/coffee, yoghurt, cereal, fruit. I'm a passionate vegetarian cook, so I may happily and spontaneously offer you something to eat, or share dinner with you, or you may ask for a meal for an additional modest fee. I have a substantial library of guidebooks to Boston and the surrounding area, so no need to schlep yours along. I don't specify a check-in/check-out time, as travelers' needs vary; I have a certain amount of flexibility which allows me to welcome guests at varied times in the day, and always try my best to accommodate their wishes. Please do note that, if I have a guest staying the night prior to your arrival, you will very likely not be able to check in before the early afternoon.	-0.200000
335	In the heart of hip JPVillage. Private 3rd floor-great base for Boston visiting: attending conferences, seeing your kids, the various hospital complexes and the sights.Plenty of bicycle kiosks as well as bus and "T" access close by.	Boston	MA	Light-filled En Suite Private Space	We view our house as a good launching spot for people that are coming to Boston to participate in an athletic event, see their children in college, go to a conference, see medical specialists at Longwood Medical Center or just see the sites.	-0.200000
374	Sunny room in apartment conveniently located in green JP: a 5 minute walk to the Stony Brook T stop, and 15 minutes ride downtown, and an abundance of shops, cafes, restaurants, moments away, together with Jamaica Pond and the Arnold Arboretum.	Boston	MA	Bed, Bath & Beyond IV in JP	Breakfast foods are provided -- tea/coffee, yoghurt, cereal, fruit. I'm a passionate vegetarian cook, so I may happily and spontaneously offer you something to eat, or share dinner with you, or you may ask for a meal for an additional modest fee. I have a substantial library of guidebooks to Boston and the surrounding area, so no need to schlep yours along. I don't specify a check-in/check-out time, as travelers' needs vary; I have a certain amount of flexibility which allows me to welcome guests at varied times in the day, and always try my best to accommodate their wishes. Please do note that, if I have a guest staying the night prior to your arrival, you will very likely not be able to check in before the early afternoon.	-0.200000
1682	Living at Vesta at West End Apartments, residents will find themselves in the center of it all surrounded by the Charles River and the Esplanade, Beacon Hill and Boston Common, the North End and the Waterfront.	Boston	MA	[1247-2C]Furnished 2BR At The Vesta	nan	-0.200000
1714	Living at Vesta at West End Apartments, residents will find themselves in the center of it all surrounded by the Charles River and the Esplanade, Beacon Hill and Boston Common, the North End and the Waterfront.	Boston	MA	[1247-1N]Furnished 1BR At The Vesta	nan	-0.200000
2294	Hi, this is a master bedroom in a 2 bd apartment at Westland Ave crossing the Mass Ave. There are kitchen and a small living room in the apartment and you need to share the bathroom with another roommate(but another room is currently empty).	Boston	MA	Large bedroom near Symphony Hall	nan	-0.175000
55	Private room in sunny spacious 3rd floor condo in Jamaica Plain, which is located approximately 12-15 mins walk from either Forest Hills Station or Green Street Orange Line T stations. The train will have you downtown Boston in approx. 15 mins. Downtown Jamaica Plain, where there are a bunch of restaurants, is also a 15 mins walk.	Boston	MA	Beautiful and sunny condo	nan	-0.171429
20	My home is a villa in one of the friendliest, and safest neighborhood of Boston. The room has a window overlooking the yard and street. All utilities and WIFI are included. Shared bathroom /kitchen. The bus line is less than a minute walk.	Boston	MA	Cozy room in a charming villa.	I speak French, Arabic and some Spanish.	-0.166667
537	(URL HIDDEN)	Boston	MA	[1857]2BR At Radian Apartments	nan	-0.166667
2399	I am AirBnBing my spare bedroom. We are walking distance to the T and a bus stop is less than 100 M away Ingres Discounts will be given to backpackers and travelers from countries I've never visited	Boston	MA	Great Room forBackpackers	nan	-0.166667
1286	Within walking distance to the Prudential Center and the Green Line.	Boston	MA	Back Bay 1 Bedroom near the Pru	nan	-0.150000
574	This apartment is in the heart of Chinatown. A minute away from Tufts Boston Campus, 5 minute walk from the T, and 15 min walk from Boston Common and Boston Public Gardens.	Boston	MA	China Town Apartment	This apartment has a dog. So must be dog friendly.	-0.150000
1941	Spacious one-bedroom in the heart of historic Beacon Hill. Steps from Boston Common and the State House. Walk to world class bars and restaurants, the waterfront, downtown, and the T.	Boston	MA	Beacon Hill 1BR near Downtown	nan	-0.150000
2693	Spacious room with private bathroom and queen size bed, 3 windows overlooking the garden and the walk to the beach behind the house. Nearby parks, 8 minutes to the Red Line then 12 minutes to downtown, 25 minutes to Cambridge, off street parking.	Boston	MA	Andrea's Private Bath T Downtown	You can walk out of the back yard to Dorchester Bay and the beach. The harbor walk bike path starts there and goes on for miles. Enjoy the fresh fruit, orange juice, english muffins, Mary's granola, and yogurt in the morning or anytime. Barney makes great coffee in the morning as well as his signature muffins! All guests have access to a full kitchen for cooking and/or enjoying meals. Feel free to play the piano; we love music in the house!	-0.133333
1561	My unit is in the Eagle Hill area of East Boston which is located close to the airport and only a few subway stops to downtown. Everything is no more than a 10 min walk away! From my door step to Back Bay is 20-25 mins!	Boston	MA	Sunny renovated 1 bedroom in friendly East Boston	nan	-0.128125
901	My location is in the heart of South End. Convenient to get to Downtown Boston,Fenway Park, MIT, Boston Medical,many other areas in the Boston surrounding.	Boston	MA	Location Location Location	No more than 2 people. I have neighbors above and below and each side. Please be respectful in avoiding noise that will disturb the neighbors. Check In Time is Flexible Check Out Time 10:00am	-0.125000
1461	5 Minutes walk to T station, close to airport and beach. 1 bedroom in a 3 bedroom unit, share kitchen and bathroom with 3 other roomates.	Boston	MA	Close to airport, downtown Boston	nan	-0.125000
1463	5 Minutes walk to T station, close to airport and beach. 1 bedroom in a 3 bedroom unit, share kitchen and bathroom with 3 other roomates.	Boston	MA	Close to the airport, 5 minutes walk to T Station	nan	-0.125000
1514	5 Minutes walk to T station, close to airport and beach. 1 bedroom in a 3 bedroom unit, share kitchen and bathroom with 3 other roomates.	Boston	MA	Boston Gladstone	nan	-0.125000
1604	Land at Logan. Short ride to Charlestown. Walk everywhere. Small but comfy bedroom with bathroom.	Boston	MA	Charlestown Brownstone	nan	-0.125000
2718	This Location is 15-minute Walk to Andrew station, which is 2 Stops from Downtown, 3 Stops From Park street (10-minute Train Ride). This Location is extremely close to Downtown.	Boston	MA	Convenient 3bd Near Downtown	nan	-0.125000
2993	Cozy 1 bedroom. Few blocks away from Castle Island and Carson Beach. A mile to Boston Convention Exhibition Center and Seaport area with oceanview restaurants/bars. Convenient buses to Copley Square, South Station and Red or Silver Lines.	Boston	MA	Private Rm w/Queen Bed & Full Bath	Depending on arrival/departure times and availability, can provide private car pick up/drop off. Please inquire for details if interested. If car parking is needed, please inquire for details.	-0.125000
1755	Fully furnished 2 bedroom apartment in historic building. Close to all that Boston has to offer: Public Garden, Boston common, MGH, Esplanade and Charles Street. Convenient to public transportation and shopping at Newbury. Not suitable for children.	Boston	MA	Well lit condo in BeaconHill Boston	This is our home! Please enjoy it.	-0.115000
1903	Boston's second oldest hotel. This is a small private guest rooms with one twin bed, private bathroom, and continental breakfast each morning. 24 hour concierge and reception. Two blocks to Boston Common, in the heart of the Theatre district.	Boston	MA	Hotel-Room w/twin bed, bathroom and free breakfast	nan	-0.110000
3326	Your home base to take Boston by storm. Situated on a peaceful street in lively Allston, this apartment scores big on location, convenience, and design.	Boston	MA	Chic 3BR in Trendy Allston	nan	-0.103409
462	Address - 15c Smith Street, Roxbury Crossing, Boston MA, 02120 Housing is exclusively for students (boys). 5 min walk to Roxbury Crossing Station (Orange Line) and Longwood Medical Area ( Green Line).	Boston	MA	Shared apt in heart of Boston.	nan	-0.100000
1554	Cozy 3rd floor 1 bedroom apartment one block from maverick Sq /T stop. One stop from Logan Airport and one stop from the heart of Boston. Fully applianced kitchen, washer dryer in unit	Boston	MA	Extended stays	nan	-0.100000
1100	This listing does not represent one room but a pool of 3 rooms under a large condominium in Downtown Boston. This mean that you may be few days in one room and few days in another room but within the same condominium.	Boston	MA	Boston South End - Multiple Rooms	Considering this listing is a pool of rooms available in a large condominium, you need to be flexible. You may be in one room for few days and in another room on another days. Every effort will be make to keep you in the same room but it will depend on the rooms demand.	-0.099643
2485	near BC shuttle, supermarket and several cafe. This spacious 2 , only 1 minute from public transportation. Room is small and cozy with a Queen size bed - 60 x 84 inches (152 cm 213 cm) . Closet is very small. Apartment is pet-free and smoke-free	Boston	MA	near BC shuttle house summer sublet	nan	-0.096429
312	Studio guest suite with private entrance, centrally located in Jamaica Plain. 2 minute walk to orange line subway train station! 日本語で､連絡が可能です｡日本人のｵｰﾅｰです｡	Boston	MA	Private studio, awesome location!!	No oven or range	-0.089286
3133	10 minute walk from Broadway Station on the red line. Very cozy living arrangement. 10 minute walk from the grocery store. Around the block from plenty of restaurants and shops. Laid back roommates.	Boston	MA	South Boston Apartment	nan	-0.086667
1460	Just a short airport shuttle ride and a one and a half block walk to my door. Centered between the Airport and Maverick T stations. 3 mins. to scenic Boston.	East Boston	MA	#3 Real close to the airport...	nan	-0.083333